﻿using System;
using System.Collections.Generic;
using System.Web;
using System.Net;
using System.IO;

using System.Text;

namespace OA.Common
{
    /// <summary>
    /// 爬虫类，抓取其他网页内容
    /// </summary>
    public class Thief
    {
        /// <summary>
        /// 使用Post方式抓取其他网页内容（web需要提交的网页地址-postData需要提交的数据）
        /// </summary>
        public static string Post(string Web, string postData)
        {
            //在发送的内容里包括中文和空格
            string str = "";
            //地址
            postData = postData.Replace(" ", "%20");//替换空格
            try
            {
                //获取URL并进行发送字串
                HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(Web);
                Stream myStream = new MemoryStream();//定义这个Stream是只是为了得到发送字串 经过编码之后得到的byte的长度。
                StreamWriter myStreamWriter = new StreamWriter(myStream, Encoding.Default);//用默认编码 得到Stream            

                myStreamWriter.Write(postData);
                myStreamWriter.Flush();
                long len = myStream.Length;//目的完成
                myStreamWriter.Close();

                myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";
                myHttpWebRequest.ContentLength = len;
                //如果字符串中存在中文 使用postData.Length得到长度和编码之
                //后的长度是不一样的:(
                myHttpWebRequest.Method = "POST";
                Stream newStream = myHttpWebRequest.GetRequestStream();
                //编码使用Encoding.Default 换来换去编码方法 还是用它得到的中文不会出现乱码,特别是把request("")得到的值写入sql数据库的时候 真是奇怪^O^
                myStreamWriter = new StreamWriter(newStream, Encoding.Default);
                myStreamWriter.Write(postData);
                myStreamWriter.Close();
                //获取本次的回馈,先创建接收回馈的类HTTPWEBRESPONSE
                HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();
                //创建接收回馈的字节流类
                Stream receiveStream = myHttpWebResponse.GetResponseStream();//得到回写的字节流 做爬虫的基础？？？
                Encoding encode;
                //encode = System.Text.Encoding.GetEncoding(myHttpWebResponse.CharacterSet.ToLower);
                switch (myHttpWebResponse.CharacterSet.ToLower())
                {
                    case "utf-8":
                        encode = System.Text.Encoding.GetEncoding("utf-8");
                        break;
                    case "gb2312":
                        encode = System.Text.Encoding.GetEncoding("gb2312");
                        break;
                    default:
                        encode = System.Text.Encoding.GetEncoding("gb2312"); //System.Text.Encoding.Default;
                        break;
                }
                // encode = System.Text.Encoding.Default;
                StreamReader readStream = new StreamReader(receiveStream, encode);//);

                str = readStream.ReadToEnd();
                readStream.Close();
                myHttpWebResponse.Close();
                return str;
            }
            catch (Exception e)
            {
                throw e;
            }
        }
        /// <summary>
        /// 使用Get方式抓取其他网页内容（url需要获取的网页地址）
        /// </summary>
        public static string Get(string url)
        {
            string tempurl = UrlEncode(url, System.Text.Encoding.GetEncoding("gb2312"));
            HttpWebRequest webr = (HttpWebRequest)WebRequest.Create(tempurl);
            HttpWebResponse wb = (HttpWebResponse)webr.GetResponse();
            Stream sr = wb.GetResponseStream();
            Encoding encode;
            //encode = System.Text.Encoding.GetEncoding(myHttpWebResponse.CharacterSet.ToLower);
            switch (wb.CharacterSet.ToLower())
            {
                case "utf-8":
                    encode = System.Text.Encoding.GetEncoding("utf-8");
                    break;
                case "gb2312":
                    encode = System.Text.Encoding.GetEncoding("gb2312");
                    break;
                default:
                    encode = System.Text.Encoding.GetEncoding("gb2312"); //System.Text.Encoding.Default;
                    break;
            }
            StreamReader sr1 = new StreamReader(sr, encode);
            string zz = sr1.ReadToEnd();
            sr1.Close();
            wb.Close();
            return zz;
        }
        /// <summary>
        /// 使用Get方式抓取RSS内容（url需要获取的RSS地址）
        /// </summary>
        public static Stream GetStreamRSS(string url)
        {
            string tempurl = UrlEncode(url, System.Text.Encoding.GetEncoding("gb2312"));
            HttpWebRequest webr = (HttpWebRequest)WebRequest.Create(tempurl);
            HttpWebResponse wb = (HttpWebResponse)webr.GetResponse();
            Stream sr = wb.GetResponseStream();
            wb.Close();
            return sr;
        }
        /// <summary>
        /// 使用Get方式抓取任意内容返回一个字节数组
        /// </summary>
        public static byte[] GetStream(string url)
        {
            string tempurl = UrlEncode(url, System.Text.Encoding.GetEncoding("gb2312"));
            byte[] by = null;
            HttpWebRequest webr = (HttpWebRequest)WebRequest.Create(tempurl);
            HttpWebResponse wb = (HttpWebResponse)webr.GetResponse();
            Stream sr = wb.GetResponseStream();
            long lo = wb.ContentLength;
            by = new byte[lo];
            int z = 0;
            long i = 0;
            while ((z = sr.ReadByte()) != -1)
            {
                by[i++] = Convert.ToByte(z);
            }
            wb.Close();
            return by;
        }
        /// <summary>
        /// 使用Get方式根据编码抓取内容（enc编码方式，例如System.Text.Encoding.GetEncoding("gb2312")）
        /// </summary>
        public static string Get(string url, Encoding enc)
        {
            string tempurl = UrlEncode(url, enc);
            HttpWebRequest webr = (HttpWebRequest)WebRequest.Create(tempurl);
            HttpWebResponse wb = (HttpWebResponse)webr.GetResponse();
            Stream sr = wb.GetResponseStream();
            StreamReader sr1 = new StreamReader(sr, enc);
            string zz = sr1.ReadToEnd();
            sr1.Close();
            wb.Close();
            return zz;
        }
        private static string UrlEncode(string url, Encoding enc)
        {
            byte[] bs = enc.GetBytes(url);
            StringBuilder sb = new StringBuilder();
            for (int i = 0; i < bs.Length; i++)
            {
                if (bs[i] < 128)
                    sb.Append((char)bs[i]);
                else
                {
                    sb.Append("%" + bs[i++].ToString("x").PadLeft(2, '0'));
                    sb.Append("%" + bs[i].ToString("x").PadLeft(2, '0'));
                }
            }
            return sb.ToString();
        }
    }
}
